{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.8.5 64-bit ('ENTER': virtualenv)",
   "metadata": {
    "interpreter": {
     "hash": "aa9e82663741a35949d10b71616b7da32b0b1a8a92bded1e278bf973221dadc2"
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras import datasets, layers, Sequential, optimizers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Downloading data from https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz\n",
      "170500096/170498071 [==============================] - 127s 1us/step\n"
     ]
    }
   ],
   "source": [
    "(x, y), (x_test, y_test) = datasets.cifar10.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(50000,), dtype=uint8, numpy=array([6, 9, 9, ..., 9, 1, 1], dtype=uint8)>"
      ]
     },
     "metadata": {},
     "execution_count": 10
    }
   ],
   "source": [
    "y = tf.squeeze(y, axis=1)\n",
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(10000,), dtype=uint8, numpy=array([3, 8, 8, ..., 5, 1, 7], dtype=uint8)>"
      ]
     },
     "metadata": {},
     "execution_count": 11
    }
   ],
   "source": [
    "y_test = tf.squeeze(y_test, axis=1)\n",
    "y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_db = tf.data.Dataset.from_tensor_slices((x, y))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(x, y):\n",
    "    x = 2 * tf.cast(x, dtype=tf.float32) / 255. - 1\n",
    "    y = tf.cast(y, dtype=tf.int32)\n",
    "    return x, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = optimizers.Adam(lr=1e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_db = train_db.shuffle(1000).map(preprocess).batch(512)\n",
    "test_db = tf.data.Dataset.from_tensor_slices((x_test, y_test))\n",
    "test_db = test_db.map(preprocess).batch(512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Model: \"sequential\"\n_________________________________________________________________\nLayer (type)                 Output Shape              Param #   \n=================================================================\nconv2d (Conv2D)              (4, 32, 32, 64)           1792      \n_________________________________________________________________\nconv2d_1 (Conv2D)            (4, 32, 32, 64)           36928     \n_________________________________________________________________\nmax_pooling2d (MaxPooling2D) (4, 16, 16, 64)           0         \n_________________________________________________________________\nconv2d_2 (Conv2D)            (4, 16, 16, 128)          73856     \n_________________________________________________________________\nconv2d_3 (Conv2D)            (4, 16, 16, 128)          147584    \n_________________________________________________________________\nmax_pooling2d_1 (MaxPooling2 (4, 8, 8, 128)            0         \n_________________________________________________________________\nconv2d_4 (Conv2D)            (4, 8, 8, 256)            295168    \n_________________________________________________________________\nconv2d_5 (Conv2D)            (4, 8, 8, 256)            590080    \n_________________________________________________________________\nmax_pooling2d_2 (MaxPooling2 (4, 4, 4, 256)            0         \n_________________________________________________________________\nconv2d_6 (Conv2D)            (4, 4, 4, 512)            1180160   \n_________________________________________________________________\nconv2d_7 (Conv2D)            (4, 4, 4, 512)            2359808   \n_________________________________________________________________\nmax_pooling2d_3 (MaxPooling2 (4, 2, 2, 512)            0         \n_________________________________________________________________\nconv2d_8 (Conv2D)            (4, 2, 2, 512)            2359808   \n_________________________________________________________________\nconv2d_9 (Conv2D)            (4, 2, 2, 512)            2359808   \n_________________________________________________________________\nmax_pooling2d_4 (MaxPooling2 (4, 1, 1, 512)            0         \n=================================================================\nTotal params: 9,404,992\nTrainable params: 9,404,992\nNon-trainable params: 0\n_________________________________________________________________\nModel: \"sequential_1\"\n_________________________________________________________________\nLayer (type)                 Output Shape              Param #   \n=================================================================\ndense (Dense)                (4, 256)                  131328    \n_________________________________________________________________\ndense_1 (Dense)              (4, 128)                  32896     \n_________________________________________________________________\ndense_2 (Dense)              (4, 10)                   1290      \n=================================================================\nTotal params: 165,514\nTrainable params: 165,514\nNon-trainable params: 0\n_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "conv_layers = [\n",
    "    layers.Conv2D(64, kernel_size=[3, 3], padding=\"same\", activation=tf.nn.relu),\n",
    "    layers.Conv2D(64, kernel_size=[3, 3], padding=\"same\", activation=tf.nn.relu),\n",
    "    layers.MaxPool2D(pool_size=[2, 2], strides=2, padding=\"same\"),\n",
    "\n",
    "    layers.Conv2D(128, kernel_size=[3, 3], padding=\"same\", activation=tf.nn.relu),\n",
    "    layers.Conv2D(128, kernel_size=[3, 3], padding=\"same\", activation=tf.nn.relu),\n",
    "    layers.MaxPool2D(pool_size=[2, 2], strides=2, padding=\"same\"),\n",
    "\n",
    "    layers.Conv2D(256, kernel_size=[3, 3], padding=\"same\", activation=tf.nn.relu),\n",
    "    layers.Conv2D(256, kernel_size=[3, 3], padding=\"same\", activation=tf.nn.relu),\n",
    "    layers.MaxPool2D(pool_size=[2, 2], strides=2, padding=\"same\"),\n",
    "\n",
    "    layers.Conv2D(512, kernel_size=[3, 3], padding=\"same\", activation=tf.nn.relu),\n",
    "    layers.Conv2D(512, kernel_size=[3, 3], padding=\"same\", activation=tf.nn.relu),\n",
    "    layers.MaxPool2D(pool_size=[2, 2], strides=2, padding=\"same\"),\n",
    "\n",
    "    layers.Conv2D(512, kernel_size=[3, 3], padding=\"same\", activation=tf.nn.relu),\n",
    "    layers.Conv2D(512, kernel_size=[3, 3], padding=\"same\", activation=tf.nn.relu),\n",
    "    layers.MaxPool2D(pool_size=[2, 2], strides=2, padding=\"same\")\n",
    "]\n",
    "\n",
    "conv_net = Sequential(conv_layers)\n",
    "\n",
    "fc_net = Sequential([\n",
    "    layers.Dense(256, activation=tf.nn.relu),\n",
    "    layers.Dense(128, activation=tf.nn.relu),\n",
    "    layers.Dense(10, activation=None)\n",
    "])\n",
    "\n",
    "conv_net.build(input_shape=[4, 32, 32, 3])\n",
    "fc_net.build(input_shape=[4, 512])\n",
    "conv_net.summary()\n",
    "fc_net.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "variables = conv_net.trainable_variables + fc_net.trainable_variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "0 0 loss: 2.3027024269104004\n"
     ]
    },
    {
     "output_type": "error",
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-25-2141ff7bf965>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      9\u001b[0m             \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlosses\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcategorical_crossentropy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_onehot\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogits\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfrom_logits\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m             \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreduce_mean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m         \u001b[0mgrads\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtape\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgradient\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvariables\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     12\u001b[0m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_gradients\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrads\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvariables\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Downloads/ENTER/lib/python3.8/site-packages/tensorflow/python/eager/backprop.py\u001b[0m in \u001b[0;36mgradient\u001b[0;34m(self, target, sources, output_gradients, unconnected_gradients)\u001b[0m\n\u001b[1;32m   1078\u001b[0m                           for x in nest.flatten(output_gradients)]\n\u001b[1;32m   1079\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1080\u001b[0;31m     flat_grad = imperative_grad.imperative_grad(\n\u001b[0m\u001b[1;32m   1081\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tape\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1082\u001b[0m         \u001b[0mflat_targets\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Downloads/ENTER/lib/python3.8/site-packages/tensorflow/python/eager/imperative_grad.py\u001b[0m in \u001b[0;36mimperative_grad\u001b[0;34m(tape, target, sources, output_gradients, sources_raw, unconnected_gradients)\u001b[0m\n\u001b[1;32m     69\u001b[0m         \"Unknown value for unconnected_gradients: %r\" % unconnected_gradients)\n\u001b[1;32m     70\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 71\u001b[0;31m   return pywrap_tfe.TFE_Py_TapeGradient(\n\u001b[0m\u001b[1;32m     72\u001b[0m       \u001b[0mtape\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tape\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     73\u001b[0m       \u001b[0mtarget\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Downloads/ENTER/lib/python3.8/site-packages/tensorflow/python/eager/backprop.py\u001b[0m in \u001b[0;36m_gradient_function\u001b[0;34m(op_name, attr_tuple, num_inputs, inputs, outputs, out_grads, skip_input_indices, forward_pass_name_scope)\u001b[0m\n\u001b[1;32m    160\u001b[0m       \u001b[0mgradient_name_scope\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mforward_pass_name_scope\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"/\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    161\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname_scope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgradient_name_scope\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 162\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mgrad_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmock_op\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mout_grads\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    163\u001b[0m   \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    164\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mgrad_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmock_op\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mout_grads\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Downloads/ENTER/lib/python3.8/site-packages/tensorflow/python/ops/nn_grad.py\u001b[0m in \u001b[0;36m_Conv2DGrad\u001b[0;34m(op, grad)\u001b[0m\n\u001b[1;32m    585\u001b[0m   \u001b[0;31m# in Eager mode.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    586\u001b[0m   return [\n\u001b[0;32m--> 587\u001b[0;31m       gen_nn_ops.conv2d_backprop_input(\n\u001b[0m\u001b[1;32m    588\u001b[0m           \u001b[0mshape_0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    589\u001b[0m           \u001b[0mop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Downloads/ENTER/lib/python3.8/site-packages/tensorflow/python/ops/gen_nn_ops.py\u001b[0m in \u001b[0;36mconv2d_backprop_input\u001b[0;34m(input_sizes, filter, out_backprop, strides, padding, use_cudnn_on_gpu, explicit_paddings, data_format, dilations, name)\u001b[0m\n\u001b[1;32m   1238\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0mtld\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_eager\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1239\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1240\u001b[0;31m       _result = pywrap_tfe.TFE_Py_FastPathExecute(\n\u001b[0m\u001b[1;32m   1241\u001b[0m         \u001b[0m_ctx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"Conv2DBackpropInput\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_sizes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfilter\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout_backprop\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1242\u001b[0m         \u001b[0;34m\"strides\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstrides\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"use_cudnn_on_gpu\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muse_cudnn_on_gpu\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"padding\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for epoch in range(50):\n",
    "    for step, (x, y) in enumerate(train_db):\n",
    "        with tf.GradientTape() as tape:\n",
    "            out = conv_net(x)\n",
    "            out = tf.reshape(out, [-1, 512])\n",
    "            logits = fc_net(out)\n",
    "\n",
    "            y_onehot = tf.one_hot(y, depth=10)\n",
    "            loss = tf.losses.categorical_crossentropy(y_onehot, logits, from_logits=True)\n",
    "            loss = tf.reduce_mean(loss)\n",
    "        grads = tape.gradient(loss, variables)\n",
    "        optimizer.apply_gradients(zip(grads, variables))\n",
    "\n",
    "        if step % 100 == 0:\n",
    "            print(epoch, step, 'loss:', float(loss))\n",
    "\n",
    "    total_num = 0\n",
    "    total_correct = 0      \n",
    "    for x, y in test_db:\n",
    "        out = conv_net(x)\n",
    "        out = tf.reshape(out, [-1, 512])\n",
    "        logits = fc_net(out)\n",
    "        prob = tf.nn.softmax(logits, axis=1)\n",
    "        pred = tf.argmax(prob, axis=1)\n",
    "        pred = tf.cast(pred, dtype=tf.int32)\n",
    "        \n",
    "        correct = tf.cast(tf.equal(pred, y), dtype=tf.int32)\n",
    "        correct = tf.reduce_sum(correct)\n",
    "\n",
    "        total_num += x.shape[0]\n",
    "        total_correct += int(correct)\n",
    "    acc = total_correct / total_num\n",
    "    print(\"epoch: \", epoch, \"acc: \", acc)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = tf.random.normal([1, 7, 7, 1])\n",
    "layer = layers.Conv2D(1, kernel_size=3, strides=1, dilation_rate=2) # 空洞卷积\n",
    "out = layer(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "TensorShape([1, 3, 3, 1])"
      ]
     },
     "metadata": {},
     "execution_count": 27
    }
   ],
   "source": [
    "out.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = tf.range(25)"
   ]
  }
 ]
}